# -*- coding: utf-8 -*-
# Author: Dinh Manh Dau <dinhmanhdau@gmail.com>
import urllib, urllib2
import re, sys, os
import datetime, time
import traceback
import urlparse
import mechanize
import HTMLParser
import binascii, hashlib
import cStringIO as StringIO
import argparse
# lib required python 2.5+
#import paramiko
import logging
#import Image
import gzip

from lxml       import etree
from urlparse   import urljoin
#from termcolor  import cprint
#from mongolog.handlers import MongoHandler
stringify = etree.XPath("string()")
os.umask(0000)

# def getMongoLog(host, port, name):
    # log = logging.getLogger(name)
    # log.setLevel(logging.WARNING)
    # log.addHandler(MongoHandler.to('mongolog', 'log', host=host, port=port))
    # return log

def cprint(text, color):
    try:
        print text
    except:
        traceback.print_exc()

def getLogger(name, file_log):
    log = logging.getLogger(name)
    log.setLevel(logging.DEBUG)
    if not file_log.endswith('log'): file_log = '{0}.log'.format(file_log)
    file_log = '/tmp/logs/{0}'.format(file_log)
    handler = logging.FileHandler(file_log)
    formater = '%(asctime)s level=%(levelname)s logger=%(name)s msg=%(message)s'
    handler.setFormatter(logging.Formatter(formater))
    log.addHandler(handler)
    return log

def getHTML(url, output=False):
    try:
        if type(url).__name__ == 'unicode': url = url.encode('utf-8')
        response = urllib.urlopen(url)
        html = response.read()
        if output: print html
        return html
    except:
        traceback.print_exc()
        
def getXMLTree2(url, isXML=False, userAgent=False, outputHTML=False, returnHTML=False, charset=None, removeTags=None, data = {}):
    try:
        ''' Hàm xây dựng cây:
            - Đọc HTML từ URL đưa vào
            - options : encoding='utf-8' override phương thức encoding
 = isXML :
                +) False : HTMLParser
                +) True  : XMLParser
        '''
        if type(url).__name__ == 'unicode': url = url.encode('utf-8')
        if userAgent:
            user_agent = ''
            if userAgent==True:
                #user_agent = 'Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10'
                #user_agent = 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7'
                user_agent = 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0';
            else:
                user_agent = userAgent
            opener = mechanize.build_opener(mechanize.HTTPRefererProcessor, mechanize.HTTPRefreshProcessor, mechanize.HTTPEquivProcessor)
            opener.addheaders = [('X-Requested-With', 'XMLHttpRequest'),
                                 ('Accept', 'application/json, text/javascript, text/html, application/xhtml+xml, application/xml, q=0.9, */*; q=0.01'),
                                 ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'),
                                 ('Accept-Encoding', 'gzip,deflate,sdch'),
                                 ("User-agent", user_agent),
                                 ('Cookie', '__utma=46011704.818903052.1356707159.1358829310.1358850727.9; __utmz=46011704.1356707159.1.1.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided); __gads=ID=339c5dd66114577b:T=1356707199:S=ALNI_MZgxZ_9mUfk4QHZgSr-kAoMj07DmA; PHPSESSID=pivu4f8f4pehodbjlftoursmj4; __utmc=46011704; location.href; popads=1; __utmb=46011704.2.10.1358850727')]
            if len(data) > 0:
                req = urllib2.Request(url, urllib.urlencode(data))
                response = urllib2.urlopen(req)
            else:
                response = opener.open(url)
        else:
            if len(data) > 0:
                req = urllib2.Request(url, urllib.urlencode(data))
                response = urllib2.urlopen(req)
            else:
                response = urllib.urlopen(url)
        '''
        if encode==None:
            html = response.read()
        else:
            parser = HTMLParser.HTMLParser()
            html = response.read().decode(encode)
            html = parser.unescape(html)
        '''
        html = response.read()
        content_type = response.info().getheader('Content-Encoding', '')
        if content_type == 'gzip':
            gzipper = gzip.GzipFile(fileobj=StringIO.StringIO(html))
            html = gzipper.read()
        if removeTags!=None:
            for tag in removeTags: html = html.replace(tag, '')
        if outputHTML: print(html)
        if isXML==False: parser = etree.HTMLParser(encoding='utf-8')
        else: parser = etree.XMLParser(encoding='utf-8')
        if charset:
            tree = etree.parse(StringIO.StringIO(html), etree.HTMLParser(encoding=charset))
        else:
            tree = etree.parse(StringIO.StringIO(html), parser)
        if returnHTML:
            return tree, html
        return tree
    except:
        traceback.print_exc()

def getXMLTree(url, isXML=False, userAgent=False, outputHTML=False, returnHTML=False, charset=None, removeTags=None, data = {}):
    try:
        ''' Hàm xây dựng cây:
            - Đọc HTML từ URL đưa vào
            - options : encoding='utf-8' override phương thức encoding
 = isXML :
                +) False : HTMLParser
                +) True  : XMLParser
        '''
        if type(url).__name__ == 'unicode': url = url.encode('utf-8')
        if userAgent:
            user_agent = ''
            if userAgent==True:
                #user_agent = 'Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10'
                user_agent = 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7'
            else:
                user_agent = userAgent
            opener = mechanize.build_opener(mechanize.HTTPRefererProcessor, mechanize.HTTPRefreshProcessor, mechanize.HTTPEquivProcessor)
            opener.addheaders = [('X-Requested-With', 'XMLHttpRequest'),
                                 ('Accept', 'application/json, text/javascript, text/html, application/xhtml+xml, application/xml, q=0.9, */*; q=0.01'),
                                 ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'),
                                 ('Accept-Encoding', 'gzip,deflate,sdch'),
                                 ("User-agent", user_agent)]
            if len(data) > 0:
                req = urllib2.Request(url, urllib.urlencode(data))
                response = urllib2.urlopen(req)
            else:
                response = opener.open(url)
        else:
            if len(data) > 0:
                req = urllib2.Request(url, urllib.urlencode(data))
                response = urllib2.urlopen(req)
            else:
                response = urllib.urlopen(url)
        '''
        if encode==None:
            html = response.read()
        else:
            parser = HTMLParser.HTMLParser()
            html = response.read().decode(encode)
            html = parser.unescape(html)
        '''
        html = response.read()
        content_type = response.info().getheader('Content-Encoding', '')
        if content_type == 'gzip':
            gzipper = gzip.GzipFile(fileobj=StringIO.StringIO(html))
            html = gzipper.read()
        if removeTags!=None:
            for tag in removeTags: html = html.replace(tag, '')
        if outputHTML: print(html)
        if isXML==False: parser = etree.HTMLParser(encoding='utf-8')
        else: parser = etree.XMLParser(encoding='utf-8')
        if charset:
            tree = etree.parse(StringIO.StringIO(html), etree.HTMLParser(encoding=charset))
        else:
            tree = etree.parse(StringIO.StringIO(html), parser)
        if returnHTML:
            return tree, html
        return tree
    except:
        traceback.print_exc()

def getListTextNode(node, _xpath):
    ''' Giá trị trả về là '' nếu không có kết quả
    '''
    result = []
    try:
        tmpsNode = node.xpath(_xpath)
        for tmpNode in tmpsNode:
            text = getStringWithNode(tmpNode)
            if text!=None and len(text)>0: result.append(text)
        return result
    except:
        traceback.print_exc()
        return result

def getTextNode(node, _xpath, type):
    ''' Giá trị trả về là '' nếu không có kết quả
    '''
    result = ''
    if type!=None and type=='int': result = 0
    try:
        tmpsNode = node.xpath(_xpath)
        if len(tmpsNode) > 0:
            _tmp = getStringWithNode(tmpsNode[0])
            tmp = toAscii(_tmp)
            if type=='int':
                return regexStr(tmp, '(\d+)', 1, 'int', 0)
            if type=='float':
                result = regexStr(tmp, '(\d+).(\d+)', 1, 'float', 0)
                if result==0:
                    result = regexStr(tmp, '(\d+)', 1, 'int', 0)
                return result
            return _tmp
    except:
        traceback.print_exc()
        return result

def regexStr(text, pattern, index, type, ret_default):
    ''' type = 'str' or 'int', 'float'
    '''
    try:
        if text==None or text=="": return ret_default
        text = toAscii(text)
        text = text.lower()
        text = ' '.join(text.split())
        regex = re.compile(pattern)
        m = regex.search(text)
        if m:
            if type=='int': return int(float(m.group(1)))
            if type=='float':
                tmp = float(m.group(2))
                if tmp < 10.0:
                    return float(m.group(1)) + tmp/10
                return float(m.group(1)) + tmp/100
        return ret_default
    except:
        traceback.print_exc()
        return ret_default

def getPathWithName(src, folder=''):
    ''' Hàm lấy tên ảnh qua xử lý md5(src)
        Giá trị trả về là chuỗi mã hóa md5 và đường dẫn (ex: a/b/c/d)
    '''
    try:
        if type(src).__name__ == 'unicode': src = src.encode('utf-8')
        md5str = hashlib.md5(src).hexdigest()
        #today = datetime.date.today()
        #current_date = today.strftime('%Y%m%d')
        #path = '{0}/{1}/{2}'.format(current_date, md5str[0], md5str[1])
        path = '{0}/{1}'.format(md5str[0], md5str[1])
        if folder != None and len(folder) > 0:
            path = '{0}/{1}/{2}'.format(folder, md5str[0], md5str[1])
        return md5str[2:], path
    except:
        traceback.print_exc()
        return None, None

def saveImageWithSCP(src, prefix, localPath, folder='', ssh=None, sftp=None, serverName=""):
    try:
        if src==None or src=='' or ssh==None or sftp==None: return None, None, None, None
        if type(src).__name__ == 'unicode': src = src.encode('utf-8')
        img_name, path = getPathWithName(src, folder)
        path_arr = path.split('/')
        if img_name==None: return None, None, None, None
        img_ext = os.path.splitext(src)[1]
        img_ext = img_ext.lower()
        if len(img_ext)>4:
            img_ext = img_ext[:4]
            if img_ext=='.jpe': img_ext = '.jpeg'
        if len(img_ext)<=1: img_ext = '.jpg'
        preg = re.compile('jpg|png|gif')
        m = preg.search(img_ext)
        if not m: img_ext = '.jpg'
        path_file = '{0}/{1}/{2}{3}'.format(localPath, path, img_name, img_ext)
        path_dir = '{0}/{1}'.format(localPath, path)
        source = '{0}/{1}/{2}{3}'.format(prefix, path, img_name, img_ext)
        file_name = '{0}.jpg'.format(img_name)
        flag = False; retry = 0; maxRetry = 3; result = 0; size = 0
        while not flag and retry < maxRetry:
            try:
                if not os.path.isfile(path_file):
                    if not os.path.exists(path_dir): os.makedirs(path_dir, 0777)
                    location, response = urllib.urlretrieve(src, path_file)
                    if not response.has_key('Content-Type'):
                        os.unlink(location); result = None
                        raise Exception('%s: content-type not exists!' % src)
                    else:
                        if not re.search('image', response['Content-Type']):
                            os.unlink(location); result = None
                            raise Exception('%s: file is not image' % src)
                    size = os.path.getsize(location)
                    if long(response['Content-Length'])!=size:
                        os.unlink(location); result = None
                        raise Exception('%s: size of file not equal file response' % src)
                    if size > 0:
                        dir = localPath
                        for item in path_arr:
                            dir = '{0}/{1}'.format(dir, item)
                            stdin, stdout, stderr = ssh.exec_command("mkdir -m 0777 {0}".format(dir))
                        remote_file = path_file
                        info = rexists(sftp, remote_file)
                        if not info:
                            flgRun = False
                            for i in range(0, 3):
                                try:
                                    sftp.put(path_file, remote_file)
                                    print 'Copy file from local to ', serverName, ': ', path_file, ' => ', remote_file
                                    flgRun = True; break
                                except:
                                   continue
                            if flgRun:
                                cprint('Lưu ảnh thành công!', 'green'); break
                            else:
                                cprint('Không upload được ảnh lên server ' + serverName, 'red'); result = None; break
                        else: cprint('Ảnh đã tồn tại trên server ' + serverName, 'red'); result = 1
                        cprint('Lưu ảnh thành công!', 'green'); break
                    else: os.unlink(location); result = None
                else:
                    size = os.path.getsize(path_file)
                    dir = localPath
                    for item in path_arr:
                        dir = '{0}/{1}'.format(dir, item)
                        stdin, stdout, stderr = ssh.exec_command("mkdir -m 0777 {0}".format(dir))
                    remote_file = path_file
                    info = rexists(sftp, remote_file)
                    if not info:
                        flgRun = False
                        for i in range(0, 3):
                            try:
                                sftp.put(path_file, remote_file)
                                print 'Copy file from local to ', serverName, ': ', path_file, ' => ', remote_file
                                flgRun = True; break
                            except:
                               continue
                        if flgRun:
                            cprint('Upload ảnh thành công!', 'green')
                        else:
                            cprint('Không upload được ảnh lên server ' + serverName, 'red'); result = None; break
                    else: cprint('Ảnh đã tồn tại trên server ' + serverName, 'red'); result = 1
                    cprint('Ảnh đã tồn tại ở local!', 'red'); break
            except:
                traceback.print_exc()
                retry += 1
        return result, source, file_name, size
    except:
        traceback.print_exc()
        return None, None, None, None

def saveImageNotHash(src, prefix, localPath, imgName=None, nameStory=None):
    try:
        if src==None or src=='': return None, None, None, None
        if type(src).__name__ == 'unicode': src = src.encode('utf-8')
        img_ext = os.path.splitext(src)[1]
        img_ext = img_ext.lower()
        if len(img_ext)>4:
            img_ext = img_ext[:4]
            if img_ext=='.jpe': img_ext = '.jpeg'
        if len(img_ext)<=1: img_ext = '.jpg'
        preg = re.compile('jpg|png|gif')
        m = preg.search(img_ext)
        if not m: img_ext = '.jpg'
        img_name = ''
        if imgName==None: img_name = src.split('/')[-1]
        else: img_name = '{0}{1}'.format(imgName, img_ext)
        img_name = img_name.lower()
        path_file = '{0}/{1}'.format(localPath, img_name)
        path_dir = localPath
        if nameStory!=None:
            source = '{0}/{1}/{2}/{3}'.format(prefix, nameStory, localPath.split('/')[-1], img_name)
        else:
            source = '{0}/{1}/{2}'.format(prefix, localPath.split('/')[-1], img_name)
        file_name = img_name
        flag = False; retry = 0; maxRetry = 3; result = 0; size = 0; location = ''
        while not flag and retry < maxRetry:
            try:
                if not os.path.isfile(path_file):
                    if not os.path.exists(path_dir): os.makedirs(path_dir, 0777)
                    location, response = urllib.urlretrieve(src, path_file)
                    if not response.has_key('Content-Type'):
                        os.unlink(location); result = None
                        #log.error('%s: content-type not exists!' % src)
                        raise Exception('%s: content-type not exists!' % src)
                    else:
                        if not re.search('image', response['Content-Type']):
                            os.unlink(location); result = None
                            #log.error('%s: file is not image' % src)
                            raise Exception('%s: file is not image' % src)
                    size = os.path.getsize(location)
                    if long(response['Content-Length'])!=size:
                        os.unlink(location); result = None
                        #log.error('%s: size of file not equal file response' % src)
                        raise Exception('%s: size of file not equal file response' % src)
                    if size > 0:
                        cprint(file_name, 'yellow')
                        cprint('Lưu ảnh thành công!', 'green');
                        break
                    else: os.unlink(location); result = None; break
                else:
                    size = os.path.getsize(path_file)
                    if size > 0:
                      cprint('Ảnh đã tồn tại !', 'red');
                      result = 1;
                    else: os.unlink(path_file); result = None
                    break
            except:
                traceback.print_exc()
                retry += 1
        return result, source, file_name, size
    except:
        traceback.print_exc()
        return None, None, None, None

def saveImageWH(src, prefix, localPath):
    ''' Hàm lưu ảnh với src, prefix, local_path
        Giá trị trả về là :
            +) result (0 chưa tồn tại, 1: ảnh đã tồn tại)
            +) source (prefix/path/file_name) Ex: /uploads/xkcn/a/b/c/d/abcd.....jpg
            +) img_name: tên ảnh là md5(src).jpg
            +) size : Dung lượng của ảnh
    '''
    try:
        width = 0; height = 0
        if src==None or src=='': return None, None, None, None, None, None
        if type(src).__name__ == 'unicode': src = src.encode('utf-8')
        img_name, path = getPathWithName(src)
        if img_name==None: return None, None, None, None, None, None
        img_ext = os.path.splitext(src)[1]
        img_ext = img_ext.lower()
        if len(img_ext)>4:
            img_ext = img_ext[:4]
            if img_ext=='.jpe': img_ext = '.jpeg'
        if len(img_ext)<=1: img_ext = '.jpg'
        preg = re.compile('jpg|png|gif')
        m = preg.search(img_ext)
        if not m: img_ext = '.jpg'
        path_file = '{0}/{1}/{2}{3}'.format(localPath, path, img_name, img_ext)
        path_dir = '{0}/{1}'.format(localPath, path)
        source = '{0}/{1}/{2}{3}'.format(prefix, path, img_name, img_ext)

        print '########### SOURCE: ', source
        file_name = '{0}{1}'.format(img_name, img_ext)
        flag = False; retry = 0; maxRetry = 3; result = 0; size = 0
        while not flag and retry < maxRetry:
            try:
                if not os.path.isfile(path_file):
                    if not os.path.exists(path_dir): os.makedirs(path_dir, 0777)
                    location, response = urllib.urlretrieve(src, path_file)
                    if not response.has_key('Content-Type'):
                        os.unlink(location); result = None
                        raise Exception('%s: content-type not exists!' % src)
                    else:
                        if not re.search('image', response['Content-Type']):
                            os.unlink(location); result = None
                            raise Exception('%s: file is not image' % src)
                    size = os.path.getsize(location)
                    if long(response['Content-Length'])!=size:
                        os.unlink(location); result = None
                        raise Exception('%s: size of file not equal file response' % src)
                    # dau's
                    #width, height = Image.open(path_file).size
                    if size > 0:
                        cprint('Lưu ảnh thành công!', 'green'); break
                    else: os.unlink(location); result = None; break
                else:
                    size = os.path.getsize(path_file)
                    # dau's
                    #width, height = Image.open(path_file).size
                    cprint('Ảnh đã tồn tại !', 'red'); result = 1; break
            except:
                traceback.print_exc()
                retry += 1
        return result, source, file_name, size, width, height
    except:
        traceback.print_exc()
        return None, None, None, None, None, None

def saveImage(src, prefix, localPath, folder=''):
    ''' Hàm lưu ảnh với src, prefix, local_path
        Giá trị trả về là :
            +) result (0 chưa tồn tại, 1: ảnh đã tồn tại)
            +) source (prefix/path/file_name) Ex: /uploads/xkcn/a/b/c/d/abcd.....jpg
            +) img_name: tên ảnh là md5(src).jpg
            +) size : Dung lượng của ảnh
    '''
    try:
        if src==None or src=='': return None, None, None, None
        if type(src).__name__ == 'unicode': src = src.encode('utf-8')
        img_name, path = getPathWithName(src, folder)
        if img_name==None: return None, None, None, None
        img_ext = os.path.splitext(src)[1]
        img_ext = img_ext.lower()
        if len(img_ext)>4:
            img_ext = img_ext[:4]
            if img_ext=='.jpe': img_ext = '.jpeg'
        if len(img_ext)<=1: img_ext = '.jpg'
        preg = re.compile('jpg|png|gif')
        m = preg.search(img_ext)
        if not m: img_ext = '.jpg'
        path_file = '{0}/{1}/{2}{3}'.format(localPath, path, img_name, img_ext)
        path_dir = '{0}/{1}'.format(localPath, path)
        source = '{0}/{1}/{2}{3}'.format(prefix, path, img_name, img_ext)
        file_name = '{0}{1}'.format(img_name, img_ext)
        flag = False; retry = 0; maxRetry = 3; result = 0; size = 0
        while not flag and retry < maxRetry:
            try:
                if not os.path.isfile(path_file):
                    if not os.path.exists(path_dir): os.makedirs(path_dir, 0777)
                    location, response = urllib.urlretrieve(src, path_file)
                    if not response.has_key('Content-Type'):
                        os.unlink(location); result = None
                        raise Exception('%s: content-type not exists!' % src)
                    else:
                        if not re.search('image', response['Content-Type']):
                            os.unlink(location); result = None
                            raise Exception('%s: file is not image' % src)
                    size = os.path.getsize(location)
                    if response.has_key('Content-Length') and long(response['Content-Length'])!=size:
                        os.unlink(location); result = None
                        raise Exception('%s: size of file not equal file response' % src)
                    if size > 0:
                        print 'Save sucess!'; break
                    else: os.unlink(location); result = None; break
                else:
                    size = os.path.getsize(path_file)
                    print 'Image was exists !'; result = 1; break
            except:
                traceback.print_exc()
                retry += 1
        return result, source, file_name, size
    except:
        traceback.print_exc()
        return None, None, None, None

def getStringWithNode(node):
    ''' Hàm lấy nội dung của một node qua strinify
        +) Loại bỏ hết nhiều whitespace, ký tự đơn: Ex: \r \n \t
    '''
    try:
        if node==None: return ''
        text = stringify(node).strip()
        text = ' '.join(text.split())
        text = encodeUTF8(text)
        text = toUnicodeDungSan(text)
        return text
    except:
        traceback.print_exc()
        return ''

def encodeUTF8(text):
    try:
        if text==None or text=='': return text
        if type(text).__name__ == 'unicode': text = text.encode('utf-8')
        return text
    except:
        traceback.print_exc()

def convertToText(text):
    ''' Hàm loại bỏ nhiều whitespace thành single whitespace, remove ký tự đơn \r \n \t
    '''
    try:
        if text==None or text=='': return
        text = ' '.join(text.split())
        return text
    except:
        traceback.print_exc()

def getMd5(text):
    try:
        if text==None or text=='': return
        text = encodeUTF8(text)
        return hashlib.md5(text).hexdigest()
    except:
        traceback.print_exc()

def getDatatypeName(nametype):
    ''' Hàm trả về tên của kiểu dữ liệu
        Ex: int, list, ...
    '''
    try:
        if nametype==None or nametype=='': return
        return type(nametype).__name__
    except:
        traceback.print_exc()

def regexString(pattern, text):
    try:
        if pattern==None or text==None or pattern=='' or text=='': return
        preg = re.compile(pattern)
        m = preg.search(text)
        if m: return m
    except:
        traceback.print_exc()

def getCharsetWithUrl(url):
    try:
        charset = 'UTF-8'
        if url==None or url=='': return charset
        opener = urllib2.urlopen(url)
        result = opener.headers.getparam('charset')
        if result!=None: charset = result
        return charset
    except:
        traceback.print_exc()

def remove_html_tags(text):
    try:
        preg = re.compile(r'<.*?>')
        return preg.sub('', text)
    except:
        traceback.print_exc()

def remove_extra_spaces(text):
    try:
        preg = re.compile(r'\s+')
        return preg.sub('', text)
    except:
        traceback.print_exc()

def toAscii(text):
    ''' Chuyển chuỗi tiếng Việt Unicode tổ hợp về dạng Ascii không dấu '''
    if text == '': return ''
    if type(text).__name__ == 'unicode': text = text.encode('utf-8')
    listPattern = ["á|à|ả|ạ|ã|â|ấ|ầ|ẩ|ậ|ẫ|ă|ắ|ằ|ẳ|ặ|ẵ", "Á|À|Ả|Ạ|Ã|Â|Ấ|Ầ|Ẩ|Ậ|Ẫ|Ă|Ắ|Ằ|Ẳ|Ặ|Ẵ",
                   "đ", "Đ", "í|ì|ỉ|ị|ĩ", "Í|Ì|Ỉ|Ị|Ĩ", "é|è|ẻ|ẹ|ẽ|ê|ế|ề|ể|ệ|ễ", "É|È|Ẻ|Ẹ|Ẽ|Ê|Ế|Ề|Ể|Ệ|Ễ",
                   "ó|ò|ỏ|ọ|õ|ô|ố|ồ|ổ|ộ|ỗ|ơ|ớ|ờ|ở|ợ|ỡ", "Ó|Ò|Ỏ|Ọ|Õ|Ô|Ố|Ồ|Ổ|Ộ|Ỗ|Ơ|Ớ|Ờ|Ở|Ợ|Ỡ",
                   "ú|ù|ủ|ụ|ũ|ư|ứ|ừ|ử|ự|ữ", "Ú|Ù|Ủ|Ụ|Ũ|Ư|Ứ|Ừ|Ử|Ự|Ữ", "ý|ỳ|ỷ|ỵ|ỹ", "Ý|Ỳ|Ỷ|Ỵ|Ỹ"]
    rep = ['a', 'A', 'd', 'D', 'i', 'I', 'e', 'E', 'o', 'O', 'u', 'U', 'y', 'Y']
    for idx in range(len(listPattern)):
        text = re.sub(listPattern[idx], rep[idx], text)
    return text

def getArgs():
    parser = argparse.ArgumentParser()
    parser.add_argument('-e', '--run', '-run', '-copy', action='store', dest='copy_file', help='Copy file to local => server.')
    args = parser.parse_args()
    return  args.copy_file

def createSSHClient(server, port, user, passwd):
    try:
        pass
        
#        client = paramiko.SSHClient()
#        client.load_system_host_keys()
#        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
#        client.connect(server, port, user, passwd)
#
#        return client
    except:
        traceback.print_exc()

def rexists(sftp, path):
    try:
        sftp.stat(path)
        return True
    except:
        return False

def forceQuit():
    try:
        print 'Finished.', datetime.datetime.now()
        pid = os.getpid(); os._exit(1); os.kill(pid, 9)
    except:
        traceback.print_exc()

unicodeDungSan = 'a á à ả ã ạ ă ắ ằ ẳ ẵ ặ â ấ ầ ẩ ẫ ậ e é è ẻ ẽ ẹ ê ế ề ể ễ ệ i í ì ỉ ĩ ị o ó ò ỏ õ ọ ô ố ồ ổ ỗ ộ ơ ớ ờ ở ỡ ợ u ú ù ủ ũ ụ ư ứ ừ ử ữ ự y ý ỳ ỷ ỹ ỵ đ A Á À Ả Ã Ạ Ă Ắ Ằ Ẳ Ẵ Ặ Â Ấ Ầ Ẩ Ẫ Ậ E É È Ẻ Ẽ Ẹ Ê Ế Ề Ể Ễ Ệ I Í Ì Ỉ Ĩ Ị O Ó Ò Ỏ Õ Ọ Ô Ố Ồ Ổ Ỗ Ộ Ơ Ớ Ờ Ở Ỡ Ợ U Ú Ù Ủ Ũ Ụ Ư Ứ Ừ Ử Ữ Ự Y Ý Ỳ Ỷ Ỹ Ỵ Đ'
unicodeToHop = 'a á à ả ã ạ ă ắ ằ ẳ ẵ ặ â ấ ầ ẩ ẫ ậ e é è ẻ ẽ ẹ ê ế ề ể ễ ệ i í ì ỉ ĩ ị o ó ò ỏ õ ọ ô ố ồ ổ ỗ ộ ơ ớ ờ ở ỡ ợ u ú ù ủ ũ ụ ư ứ ừ ử ữ ự y ý ỳ ỷ ỹ ỵ đ A Á À Ả Ã Ạ Ă Ắ Ằ Ẳ Ẵ Ặ Â Ấ Ầ Ẩ Ẫ Ậ E É È Ẻ Ẽ Ẹ Ê Ế Ề Ể Ễ Ệ I Í Ì Ỉ Ĩ Ị O Ó Ò Ỏ Õ Ọ Ô Ố Ồ Ổ Ỗ Ộ Ơ Ớ Ờ Ở Ỡ Ợ U Ú Ù Ủ Ũ Ụ Ư Ứ Ừ Ử Ữ Ự Y Ý Ỳ Ỷ Ỹ Ỵ Đ'
aDungSan = unicodeDungSan.split(' ')
aToHop = unicodeToHop.split(' ')
def toUnicodeDungSan(strInp):
    res = strInp
    if type(res).__name__=='unicode': res = res.encode('utf-8')
    for i in range(len(aToHop)):
        res = res.replace(aToHop[i], aDungSan[i])
    return res


def getCategory():
    lstCat = {  'vanhoaxahoi': u'Văn hóa - Xã hội',
                'thegioi': u'Thế giới',
                'thethao': u'Thể thao',
                'giaitri': u'Giải trí',
                'giaoduc': u'Giáo dục',
                'phapluat': u'Pháp luật',
                'suckhoegioitinh': u'Sức khỏe - Giới tính',
                'khoahoc': u'Khoa học',
                'congngheso': u'Công nghệ số',
                'lamdepthoitrang': u'Làm đẹp - Thời trang',
                'kinhdoanh': u'Kinh doanh',
                'tieudung': u'Tiêu dùng',
                'amthuc':  u'Ẩm thực',
                'chuyenla': u'Chuyện lạ',
                'tamsu': u'Tâm sự',
             }
    return lstCat
